package edu.hawaii.webspider;


import java.util.List;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.GregorianCalendar;
import com.meterware.httpunit.WebLink;
import com.meterware.httpunit.WebRequest;
import com.meterware.httpunit.WebResponse;
import com.meterware.httpunit.HttpUnitOptions;
import com.meterware.httpunit.WebConversation;
import com.meterware.httpunit.GetMethodWebRequest;

/**
 * A demonstration class illustrating how to retrieve and process web pages using HttpUnit.
 * 
 * @author Philip Johnson
 */
public class WebSpiderExample {

  /** Contains the starting url for the crawl. */
  protected final String startUrl;
  
  @SuppressWarnings("unchecked")
  private final List<String> list = new ArrayList<String>();
  @SuppressWarnings("unchecked")
  private final List<String> pList = new ArrayList<String>();
  
  private static int numberOfLinks = 0;
  private int numberOfPages = 0;

  /**
   * A WebSpider crawls the web and returns info.
   * 
   * @param startUrl The home url for the crawl.
   */
  public WebSpiderExample(String startUrl) {
    this.startUrl = startUrl;
  }
 
  /**
   * Returns the number of links found at the startUrl.
   * 
   * @return The number of links.
   * @throws Exception if problems occur retrieving the startUrl.
   */
  public int getNumLinks() throws Exception {
    // create the conversation object which will maintain state for us
    WebConversation wc = new WebConversation();
    WebRequest request = new GetMethodWebRequest(this.startUrl);
    WebResponse response = wc.getResponse(request);
    return response.getLinks().length;
  }
  
  /**
   * Returns the number of links while crawling the first 100 pages.
   * 
   * @return The number of links.
   * @param urlLink String
   * @param numLink int
   * @throws Exception if problems.
   */
  public int traversePages(String urlLink, int numLink) throws Exception {
    HttpUnitOptions.setExceptionsThrownOnScriptError(false);
    HttpUnitOptions.setScriptingEnabled(false);
    WebConversation wc = new WebConversation();
    WebRequest request = new GetMethodWebRequest(urlLink);
    WebResponse response = wc.getResponse(request);
    WebLink[] links = response.getLinks();
    WebResponse currentPage = wc.getCurrentPage();
     
    if (numberOfPages == numLink) {
      return 1;
    }
    
    else if (numberOfPages >= 0 && numberOfPages < numLink) {
      boolean isEqual = true;
      for (int i = 0; i < list.toArray().length; i++) {
        if (list.toArray()[i].toString().equals(currentPage.getURL().toString())) {
          pList.add(currentPage.getURL().toString());
          isEqual = false;
        }
      }
      if (isEqual) {
        list.add(currentPage.getURL().toString());
        numberOfPages++;
        numberOfLinks = numberOfLinks + links.length;
        
        System.out.println();
        
        for (int j = 0; j < links.length; j++) {
          try {
            links[j].click();
          }
          catch (Exception e) {
            assert (true);
          }
          
          currentPage = wc.getCurrentPage();
          if (currentPage.isHTML() && numberOfPages < numLink) {
            try {
              traversePages(currentPage.getURL().toString(), numLink);
            }
            catch (Exception e) {
              throw new Exception(e);
            }
          }
          else {
            return numberOfLinks;
          } 
        }
      }
    }
    return numberOfLinks;
  }
  
  /**
   * Find the most popular page.
   * @return String
   */
  public String findMostPopularPage() {
    int mostVisited = 0;
    int tempMostVisited = 0;
    String mostVisitedPages = "";
    String tempMostVisitedPages = "";
    
    for (int i = 0; i < pList.toArray().length; i++) {
      for (int j = i + 1; j < pList.toArray().length; j++) {
        if (pList.get(i).equals(pList.get(j))) {
          tempMostVisited++;
          tempMostVisitedPages = pList.get(i);
        }
      }
      if (tempMostVisited > mostVisited) {
        mostVisitedPages = tempMostVisitedPages;
      }
      tempMostVisited = 0;
      tempMostVisitedPages = "";
    }
    return mostVisitedPages;
  }
  
  /**
   * Find the total pages that point to the most popular page.
   * @return int
   */
  public int findTotalPages() {
    int mostVisited = 0;
    int tempMostVisited = 0;
    
    for (int i = 0; i < pList.toArray().length; i++) {
      for (int j = i + 1; j < pList.toArray().length; j++) {
        if (pList.get(i).equals(pList.get(j))) {
          tempMostVisited++;
        }
      }
      if (tempMostVisited > mostVisited) {
        mostVisited = tempMostVisited;
      }
      tempMostVisited = 0;
    }
    return mostVisited;
  }
  
  /**
   * Prints a log list.
   * 
   * @param urlLink String
   * @param numLink int
   * @param logging String
   * @throws Exception if problems.
   */
  public void logList (String urlLink, int numLink, String logging) throws Exception {
    HttpUnitOptions.setExceptionsThrownOnScriptError(false);
    HttpUnitOptions.setScriptingEnabled(false);
    WebConversation wc = new WebConversation();
    WebRequest request = new GetMethodWebRequest(urlLink);
    WebResponse response = wc.getResponse(request);
    WebLink[] links = response.getLinks();
    WebResponse currentPage = wc.getCurrentPage();
    
    Calendar cal = new GregorianCalendar();
    int hour = cal.get(Calendar.HOUR_OF_DAY);
    int min = cal.get(Calendar.MINUTE);
    int sec = cal.get(Calendar.SECOND);
     
    if (numberOfPages == numLink) {
      numberOfPages = -1;
    }
    
    else if (numberOfPages >= 0 && numberOfPages < numLink) {
      boolean isEqual = true;
      for (int i = 0; i < list.toArray().length; i++) {
        if (list.toArray()[i].toString().equals(currentPage.getURL().toString())) {
          pList.add(currentPage.getURL().toString());
          isEqual = false;
        }
      }
      if (isEqual) {
        list.add(currentPage.getURL().toString());
        numberOfPages++;
        numberOfLinks = numberOfLinks + links.length;
        
        System.out.println(hour + ":" + min + ":" + sec + " Page: " + numberOfPages);
        System.out.println(hour + ":" + min + ":" + sec + " Retrieving " 
                           + currentPage.getURL().toString());
        System.out.println(hour + ":" + min + ":" + sec + " Found " + links.length + " links.");
        System.out.println("Total links so far: " + numberOfLinks);
        System.out.println();
    
        for (int j = 0; j < links.length; j++) {
          try {
            links[j].click();
          }
          catch (Exception e) {
            assert (true);
          }
          
          currentPage = wc.getCurrentPage();
          if (currentPage.isHTML() && numberOfPages < numLink) {
            try {
              logList(currentPage.getURL().toString(), numLink, logging);
            }
            catch (Exception e) {
              throw new Exception(e);
            }
          }
          else {
            return;
          } 
        }
      }
      else {
        return;
      }
    }
  }

  /**
   * Retrieves the httpunit home page and counts the number of links on it.
   * 
   * @param args Ignored.
   * @throws Exception If problems occur.
   */
  public static void main(String[] args) throws Exception {
    try {
      if (args.length == 0) {
        WebSpiderExample exa = new WebSpiderExample("http://hackystat.org/hackyDevSite/home.do");
        System.out.println("The HttpUnit home page contains " + exa.getNumLinks() + " links.");
      }
      else if (args.length >= 1 && args.length <= 3) {
        if ((args[0].equals("-totallinks")) &&
           ((args[1].startsWith("http",0)) || (args[1].startsWith("www", 0))) &&
           (WebSpiderExample.isNum(args[2]))) {
          WebSpiderExample crawl = new WebSpiderExample(args[1]);
          int initalNumOfLinks = Integer.parseInt(args[2]);
          int totalNumOfPages = crawl.traversePages(args[1], initalNumOfLinks);
          System.out.println("The total number of links  = " + totalNumOfPages);
        }
        else if ((args[0].equals("-mostpopular")) &&
                ((args[1].startsWith("http",0)) || (args[1].startsWith("www", 0))) &&
                (WebSpiderExample.isNum(args[2]))) {
         WebSpiderExample crawl = new WebSpiderExample(args[1]);
         int initalNumOfLinks = Integer.parseInt(args[2]);
         crawl.traversePages(args[1], initalNumOfLinks);
         String mostPopularPage = crawl.findMostPopularPage();
         int totalPages  = crawl.findTotalPages();
         System.out.println("The most popular page discovered is: ");
         System.out.println(mostPopularPage);
         System.out.println("with " + totalPages + " pages that point to it.");
       }
      }
      else if (args.length == 4) {
        if (((args[0].equals("-mostpopular")) || (args[0].equals("-totallinks"))) &&
            ((args[1].startsWith("http",0)) || (args[1].startsWith("www", 0))) &&
            (WebSpiderExample.isNum(args[2])) && (args[3].equals("-logging"))) {
          WebSpiderExample crawl = new WebSpiderExample(args[1]);
          int initalNumOfLinks = Integer.parseInt(args[2]);
          crawl.logList(args[1], initalNumOfLinks, args[3]);
        }
        else {
          WebSpiderExample exa = new WebSpiderExample("http://hackystat.org/hackyDevSite/home.do");
          System.out.println("The HttpUnit home page contains " + exa.getNumLinks() + " links.");
        }
      }
    }
    catch (ArrayIndexOutOfBoundsException e) {
      System.out.println("Array Index: " + e);
    }
  }
  
  /**
   * Retrieves the httpunit home page and counts the number of links on it.
   * 
   * @param s String
   * @return Boolean
   */
  static boolean isNum(String s) {
    String validChars = "0123456789";
    boolean isNumber = true;
  
    for (int i = 0; i < s.length() && isNumber; i++) { 
      char c = s.charAt(i); 
      if (validChars.indexOf(c) == -1) {
        isNumber = false;
      }
      else {
        isNumber = true;
      }
    }
    return isNumber;
  }
}
